package com.spider.service;

import java.util.ArrayList;
import java.util.List;

import org.apache.hadoop.hbase.client.Result;

import com.spider.dao.HBaseDAOImp;
import com.spider.dao.HBaseDao;
import com.spider.utils.ResolveHtmlUtils;
/**
 * URL去重
 * @author pns
 *
 */
public class ProcessUrl {
	
	private HBaseDao dao = new HBaseDAOImp();
	
	public static void main(String[] args) {
		ProcessUrl process = new ProcessUrl();
		ResolveHtmlUtils htmlResolve = new ResolveHtmlUtils();
//		Result result = process.dao.getRow("Test","63727281");
//		System.out.println(new String(result.getValue("base".getBytes(), "content".getBytes())));
//		htmlResolve.resolve(new File("./a"));
//		PrintWriter pw = null;
//		try {
//			pw = new PrintWriter(new FileWriter("./a"));
//		} catch (IOException e) {
//			// TODO Auto-generated catch block
//			e.printStackTrace();
//		}
		List<Result> urlList = process.dao.getRows("url", "url");
		List<Result> urledList = process.dao.getRows("url", "urled");	
		List<String> urlListFor = new ArrayList();
		List<String> urledListFor = new ArrayList();
		for(int i = 0;i < 153324;i++) {
			String content = new String(urlList.get(i).getValue("urled".getBytes(), "url".getBytes()));
			System.err.println(i);
			urlListFor.add(content);
		}
		
		for(int i = 0;i < urledList.size();i++) {
			String content = new String(urledList.get(i).getValue("url".getBytes(), "urled".getBytes()));
			System.out.println(i);
			urledListFor.add(content);
		}
		
		urlListFor.removeAll(urledListFor);
		System.out.println(urlListFor.size());
		
		
//		List<String> strList = new ArrayList();
//		for(int i = 0;i < resultList.size();i++) {
//			String content = new String(resultList.get(i).getValue("base".getBytes(), "content".getBytes()));
//			System.out.println(content);
			//			pw.write(content);
//			pw.flush();
//			pw.close();
//		}
	
//		List<Result> resultList = process.dao.getRows("html", "content");
//		List<Result> resultList2 = process.dao.getRows("html0714", "content");
//		for(int i = 0;i < resultList.size();i++) {
//			String html = new String(resultList.get(i).getValue("base".getBytes(), "content".getBytes()));
//			htmlResolve.resolve(html);
//		}
//		for(int i = 0;i < resultList2.size();i++) {
//			String html = new String(resultList2.get(i).getValue("base".getBytes(), "content".getBytes()));
//			htmlResolve.resolve(html);
//		}
	}
}
